In [1]:
%%configure
{ "name":"Spark-to-Cosmos_DB_Change_Feed_Connector", 
  "executorMemory": "8G", 
  "executorCores": 2, 
  "numExecutors": 2, 
  "driverCores": 2,
  "jars": ["wasb:///example/jars/azure-cosmosdb-spark_2.4.0_2.11-1.3.4-uber.jar", "wasb:///example/jars/azure-documentdb-1.13.0.jar", "wasb:///example/jars/azure-documentdb-rx-0.9.0-rc2.jar", "wasb:///example/jars/json-20140107.jar", "wasb:///example/jars/rxjava-1.3.0.jar", "wasb:///example/jars/rxnetty-0.4.20.jar"],
 "conf": {
    "spark.jars.excludes": "org.scala-lang:scala-reflect"
   }
}
Current session configs: {u'kind': 'pyspark', u'name': u'Spark-to-Cosmos_DB_Change_Feed_Connector', u'numExecutors': 2, u'conf': {u'spark.jars.excludes': u'org.scala-lang:scala-reflect'}, u'executorCores': 2, u'driverCores': 2, u'jars': [u'wasb:///example/jars/azure-cosmosdb-spark_2.4.0_2.11-1.3.4-uber.jar', u'wasb:///example/jars/azure-documentdb-1.13.0.jar', u'wasb:///example/jars/azure-documentdb-rx-0.9.0-rc2.jar', u'wasb:///example/jars/json-20140107.jar', u'wasb:///example/jars/rxjava-1.3.0.jar', u'wasb:///example/jars/rxnetty-0.4.20.jar'], u'executorMemory': u'8G'}
No active sessions.
In [2]:
database = "dc1_country_db"
collection = "states"

statesConfig = {
"Endpoint" : "https://dc1-cosmos-db.documents.azure.com:443/",
"Masterkey" : "lo1eKao8FkZ9nFf6aQ4lxuG2nuRvCaGRgEwmKIk5j4rJDQbR05VMCPdqtm0A9iQXOXKhXKEQFbmfl20BtSEZWw==",
"Database" : database,
"Collection" : collection, 
"preferredRegions" : "West US",
"SamplingRatio" : "1.0",
"schema_samplesize" : "200000",
"query_custom" : "SELECT c.state_code, c.state_name FROM c"
}

Starting Spark application
IDYARN Application IDKindStateSpark UIDriver logCurrent session?
9application_1576339111566_0013pysparkidleLinkLink✔
SparkSession available as 'spark'.
In [3]:
states = spark.read.format("com.microsoft.azure.cosmosdb.spark").options(**statesConfig).load()
In [4]:
states.createOrReplaceTempView("states")
states.cache()

DataFrame[state_code: string, state_name: string]
In [5]:
%%sql
select count(1) from states
count(1)
0 2
In [6]:
%%sql
select * from states
state_code state_name
0 CA California
1 AZ Arizona
In [ ]: